package main

import (
	"net/http"
	"log"
	"io/ioutil"
	"fmt"
	"bufio"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
	"regexp"
)

var (
	cityUrlReg = regexp.MustCompile(`<a href="(http://www.zhenai.com/zhenghun/[0-9a-zA-Z]+)"
									class="">([^<]+)</a>`)
)
func main(){
	const url = "http://www.zhenai.com/zhenghun"
	request, err := http.NewRequest(http.MethodGet, url, nil)
	// 遇到错误 直接退出就好了
	if err != nil {
		log.Fatalf("error %s \n",err )
	}

	// 设置浏览器的 UA
	request.Header.Add("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36")
	// todo add more request  info
	client := http.Client{}


	resp, err := client.Do(request)

	if err != nil {
		log.Fatalf("request %s but error %+v \n", url , err .Error())
	}

	// 关闭 输入流
	defer resp.Body.Close()

	newReader := bufio.NewReader(resp.Body)
	uEncoding := guessEncding(newReader)
	// 获取 文本的 输入流
	uEncodingReader := transform.NewReader(newReader, uEncoding.NewDecoder())
	data, err := ioutil.ReadAll(uEncodingReader)

//	fmt.Println("data = ",string(data ))

	matches:= cityUrlReg.FindAllSubmatch(data,-1)

	for _ , va :=range matches{
		fmt.Printf("city url is %s \n", string(va[1]))
	}

}


// 从输入流中 尽可能的获取 解码方式, 默认是 utf-8
func guessEncding(r *bufio.Reader)(uEncoding encoding.Encoding){

	data, err := r.Peek(1024)

	if err != nil {
		uEncoding = unicode.UTF8
		return uEncoding
	}

	// get text encoding , but  may be uncertain
	uEncoding, _, _ = charset.DetermineEncoding(data, "")
	return uEncoding
}
